Logistic Regression

In [13]:

    

import numpy as np
import matplotlib.pyplot as plt


    

In [14]:

    

def sigmoid(scores):
    return 1 / (1 + np.exp(-scores))


    

In [15]:

    

def log_likelihood(features, target, weights):
    scores = np.dot(features, weights)
    ll = np.sum( target*scores - np.log(1 + np.exp(scores)) )
    return ll


    

In [16]:

    

def logistic_regression(features, target, num_steps, learning_rate, add_intercept = False):
    if add_intercept:
        intercept = np.ones((features.shape[0], 1))
        features = np.hstack((intercept, features))
        
    weights = np.zeros(features.shape[1])
    
    for step in range(num_steps):
        scores = np.dot(features, weights)
        predictions = sigmoid(scores)

        # Update weights with gradient
        output_error_signal = target - predictions
        gradient = np.dot(features.T, output_error_signal)
        weights += learning_rate * gradient
        
        # Print log-likelihood every so often
        if step % 10000 == 0:
            print(log_likelihood(features, target, weights))
        
    return weights


    

In [17]:

    

np.random.seed(12)
num_observations = 5000

x1 = np.random.multivariate_normal([0, 0], [[1, .75],[.75, 1]], num_observations)
x2 = np.random.multivariate_normal([1, 4], [[1, .75],[.75, 1]], num_observations)

simulated_separableish_features = np.vstack((x1, x2)).astype(np.float32)
simulated_labels = np.hstack((np.zeros(num_observations),
                              np.ones(num_observations)))


    

In [18]:

    

weights = logistic_regression(simulated_separableish_features, simulated_labels,
                     num_steps = 300000, learning_rate = 5e-5, add_intercept=True)


    

    




-4346.26477915
-148.706722768
-142.964936231
-141.545303072
-141.060319659
-140.870315859
-140.790259128
-140.755050023
-140.739145032
-140.731834989
-140.728436667
-140.726844713
-140.72609509
-140.725740861
-140.725573069
-140.725493458
-140.725455643
-140.725437666
-140.725429116
-140.725425048
-140.725423112
-140.72542219
-140.725421751
-140.725421543
-140.725421443
-140.725421396
-140.725421373
-140.725421362
-140.725421357
-140.725421355

In [19]:

    

from sklearn.linear_model import LogisticRegression

clf = LogisticRegression(fit_intercept=True, C = 1e15)
clf.fit(simulated_separableish_features, simulated_labels)

print(clf.intercept_, clf.coef_)
print(weights)


    

    




[-13.99400797] [[-5.02712572  8.23286799]]
[-14.09225541  -5.05899648   8.28955762]